package com.lzx;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;

public class IPPool {
    public static void main(String[] args) {
        try{
            for (int i = 1;i < 10;i++){
                String url = "https://www.kuaidaili.com/free/inha/" + i;
                List<IPEntity> ips = getIP(url);
                testIP(ips);
                System.out.println(ips);
                Thread.sleep(1000*60);
            }
        }catch (Exception e){
            e.printStackTrace();
        }

    }
    public static List<IPEntity>  getIP(String url){
        List<IPEntity> ips = new LinkedList<IPEntity>();
        CloseableHttpResponse response = null;
        CloseableHttpClient httpClient = null;
        try{

            HttpGet get = new HttpGet(url);
            get.setHeader("user-agent"," Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36");
            httpClient = HttpClientBuilder.create().build();
            response = httpClient.execute(get);

            if(response.getStatusLine().getStatusCode() == 200) {
                HttpEntity entity = response.getEntity();
                String html = EntityUtils.toString(entity,"gb2312");
                Document doc = Jsoup.parse(html);
                Elements ulList = doc.select("table");
                Elements liList = ulList.get(0).select("tr");
                for(int i = 1;i < liList.size();i++) {
                    String [] strings = liList.get(i).text().split(" ");
                    IPEntity ip = new IPEntity(strings[0],new Integer(strings[1]),strings[3]);
                    ips.add(ip);
                }
                EntityUtils.consume(response.getEntity());
            }else{
                EntityUtils.consume(response.getEntity());
            }

        }catch (Exception e){
            e.printStackTrace();
        }finally {
            try{
                if(response != null){
                    response.close();
                }
                if(httpClient != null){
                    httpClient.close();
                }
            }catch (Exception e){
                e.printStackTrace();
            }
        }
        return ips;
    }
    public static List<IPEntity> testIP(List<IPEntity> ips){
        for(int i = 0;i < ips.size();i++){
            if(!CrawlerUtil.runTest(ips.get(i))){
                System.out.println("无效!  "+ips.get(i));
                ips.remove(i);
            }else {
                System.out.println("有效!  "+ips.get(i));

            }
        }
        return ips;
    }

//    public static String getHtmlByHttpURLConnection(String urlString){
//        StringBuffer stringBuffer = new StringBuffer();
//        if(!urlString.startsWith("http")){
//            urlString = "https://" + urlString;
//        }
//        InputStreamReader inputStreamReader = null;
//        BufferedReader bufferedReader = null;
//        HttpURLConnection httpURLConnection = null;
//        try{
//            URL url = new URL(urlString);
//            httpURLConnection = (HttpURLConnection)url.openConnection();
//            httpURLConnection.setRequestProperty("User-Agent",
//                    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
//            httpURLConnection.setReadTimeout(2000);
//            httpURLConnection.setRequestMethod("GET");
//            httpURLConnection.setConnectTimeout(2000);
//            if(httpURLConnection.getResponseCode() == 200){
//                inputStreamReader = new InputStreamReader(httpURLConnection.getInputStream(),"utf-8");
//
//                bufferedReader = new BufferedReader(inputStreamReader);
//
//                String line = "";
//
//                while((line = bufferedReader.readLine()) != null ){
//                    stringBuffer.append(line);
//                    stringBuffer.append("\n");
//                }
//            }else{
//                System.out.println(httpURLConnection.getResponseCode());
//            }
//
//        }catch (Exception e) {
//            e.printStackTrace();
//        }finally {
//            try{
//
//                if (bufferedReader != null){
//                    bufferedReader.close();
//                }
//                if(inputStreamReader != null){
//                    inputStreamReader.close();
//                }
//            }catch (IOException ee){
//                ee.printStackTrace();
//            }
//        }
//        return stringBuffer.toString();
//    }



}

